package com.ideal.gpt.app.docqajob;

import java.io.File;
import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;

import com.ideal.gpt.GptJobConfig;
import com.ideal.gpt.ability.docqa.caller.DocQaCaller;
import com.ideal.gpt.ability.docqa.splitter.ResponseSplitter;
import com.ideal.gpt.ability.docqa.splitter.ResponseSplitter.SplitResult;
import com.lzwork.gpt.utility.caller.GptCallResult;
import com.lzwork.gpt.utility.common.Passage;
import com.lzwork.gpt.utility.common.Passage.CutResult;
import com.lzwork.utility.TimeTicker;
import com.lzwork.utility.collection.CollUtility;
import com.lzwork.utility.data.NumberParse;
import com.lzwork.utility.file.FileUtility;
import com.lzwork.utility.file.LineContent;
import com.lzwork.utility.logger.LOGGER;
import com.lzwork.utility.officeparser.ExcelReader;
import com.lzwork.utility.officeparser.ExcelReader.Line;
import com.lzwork.utility.officeparser.ExcelReader.Page;
import com.lzwork.utility.officeparser.ExcelReader.XlsData;
import com.lzwork.utility.officeparser.ExcelSaver;
import com.lzwork.utility.officeparser.excel.StyleGroup;
import com.lzwork.utility.officeparser.excel.StyleGroup.BaseStyle;
import com.lzwork.utility.pair.StringPair;
import com.lzwork.utility.str.StrUtility;
import com.lzwork.web.RemoteCaller.CallResult;

/**
 * @author LiZheng
 * @date 2023年8月29日 上午10:59:54
 */

public class QaParseJob extends ExcelSaver
{
	public enum QuestionType
	{
		QA(1),
		Choice1(2),
		TF(3),
		ChoiceM(4),
		Fill(5),
		KeyWord(100),
		Unknown(-1);

		int bit;

		private QuestionType(int bit)
		{
			this.bit = bit;
		}

		public int bit()
		{
			return bit;
		}

		public static QuestionType parseBit(int bit)
		{
			for (QuestionType t : QuestionType.values())
			{
				if (t.bit == bit)
					return t;
			}

			return Unknown;
		}
	}

	final static String KEY_NAME = "File";
	final static String KEY_HASH = "Hash";
	final static String KEY_CONTENT = "Content";
	final static String KEY_RESPONSE = "Response";
	final static String KEY_SPLIT = "Splits";
	final static String KEY_LEN = "Len";
	final static String KEY_Q = "Question";
	final static String KEY_A = "Answer";
	final static String KEY_COST = "Cost";

	final static String TAB_SHEET = "QaParse";
	final static String TAB_INFO = "Info";

	final static String INFO_NUM = "每段落问题数";
	final static String INFO_LEN = "段落切分长度";

	public static class PerfixRemover
	{
		public PerfixRemover(File data)
		{
			List<String> lines = LineContent.loadFile(data);

			for (String line : lines)
			{
				String[] parts = line.trim().split(" ");

				String first = parts[0];
				if (!StrUtility.stringValid(first))
					continue;

				String second = CollUtility.arrayGet(parts, 1, "");
				replacer.add(new StringPair(first, second));
			}

			replacer.sort((Comparator<StringPair>) (o1, o2) -> o2.first().length() - o1.first().length());
		}

		List<StringPair> replacer = new ArrayList<>();

		public String replace(String input)
		{
			String result = input;

			for (StringPair sp : replacer)
			{
				if (input.startsWith(sp.getKey()))
					return input.replace(sp.getKey(), sp.getValue());
			}

			return result;
		}

		public StringPair replacePair(StringPair input)
		{
			String first = replace(input.first());
			String second = replace(input.second());

			return new StringPair(first, second);
		}
	}

	public static class JobInfo
	{
		String url;
		int num = 3;
		int maxLen = 5000;
		int parseLimit = -1;
		File logFile;
		boolean breatNetException = false;
	}

	QaParseJob(JobInfo info)
	{
		this.info = info;
		caller = new DocQaCaller(info.url);
	}

	JobInfo info;
	DocQaCaller caller;

	List<String> logList = new ArrayList<>();
	List<String> errLog = new ArrayList<>();

	public List<String> getErr()
	{
		return errLog;
	}

	SimpleDateFormat dateFormat = new SimpleDateFormat(StrUtility.DEF_DATE_FORMAT_LOG);

	void appendLog(String log)
	{
		appendLog(log, false);
	}

	void appendLogs(Collection<? extends String> log)
	{
		appendLogs(log, false);
	}

	void appendErrLog(Collection<? extends String> logs)
	{
		for (String str : logs)
		{
			appendErrLog(str);
		}
	}

	void appendErrLog(String log)
	{
		String line = appendLog(log, true);
		errLog.add(line);
	}

	String appendLog(String log, boolean autoSave)
	{
		String date = dateFormat.format(new Date(System.currentTimeMillis()));

		String line = date + "\t" + log;
		logList.add(line);

		if (logList.size() > logBuff && autoSave)
			flushLog();

		return line;
	}

	List<String> appendLogs(Collection<? extends String> logs, boolean autoSave)
	{
		String date = dateFormat.format(new Date(System.currentTimeMillis()));
		List<String> result = new ArrayList<>();

		for (String log : logs)
		{
			String line = date + "\t" + log;
			logList.add(line);
			result.add(line);
		}

		if (logList.size() > logBuff && autoSave)
			flushLog();

		return result;
	}

	void flushLog()
	{
		if (logList.isEmpty())
			return;

		if (info != null && info.logFile != null)
		{
			LineContent.saveFile(info.logFile, logList, true, false, "\r\n");
			LOGGER.log(LOGGER.FILE, "Flush " + logList.size() + " log lines.");
		}

		logList.clear();
	}

	class Item
	{
		String name;
		String hash;
		String size;
		List<SubItem> subItems = new ArrayList<>();

		@Override
		public String toString()
		{
			return "[F " + size + "][P " + subItems.size() + "]" + name;
		}
	}

	class SubItem
	{
		public SubItem(Item owner)
		{
			this.owner = owner;
			index = owner.subItems.size();
		}

		Item owner;
		int index;
		String content;
		String response;
		String cost;
		String state;
		List<StringPair> qaList = new ArrayList<>();

		@Override
		public String toString()
		{
			String lenStr = NumberParse.numToKiloCount(content.length());
			return "[" + (index + 1) + "/" + owner.subItems.size() + "][" + lenStr + "] " + owner.name;
		}

		void remotecall(int type, int num, String desc)
		{
			if (skipParse)
			{
				LOGGER.log("Skip " + desc);

				if (qaList.isEmpty())
				{
					fillEmptySp(num, qaList);
					cost = "Skip";
					state = "Skip";
				}

				return;
			}

			GptCallResult sp = caller.callParse(content, type, num, desc, true);

			appendErrLog(sp.getErrInfos());

			String callErr = sp.getErrType();
			if (StrUtility.stringValid(callErr))
			{
				if (callErr.equals(CallResult.UNKNOWN_ERR))
					state = sp.getErrInfos().get(0);
				else
					state = callErr;

				fillEmptySp(num, qaList);
				return;
			}

			response = sp.getResponse();
			cost = sp.getGpuCost();

			qaList = splitContent(type, response, info.num, desc);
			checkState(num);
		}

		void checkState(int num)
		{
			int count = 0;
			for (StringPair sp : qaList)
			{
				if (sp.contentValid(false))
					count++;
			}

			if (count == 0)
				state = "Empty";
			else if (count == num)
				state = "Full";
			else
				state = count + "/" + num;
		}
	}

	Map<String, Item> itemMap = new LinkedHashMap<>();
	int writeCount = 0;

	void parseFromForm(int type, File input, File output)
	{
		XlsData pages = ExcelReader.read(input);
		List<Item> toWrite = new ArrayList<>();

		appendLog("************************");
		appendLog("Form parse job for [" + input.getPath() + "]");
		appendLog("Output to: " + output.getParent());

		for (Page p : pages)
		{
			for (Line l : p.getLines())
			{
				Item item = new Item();
				item.name = l.getDataAt(0);
				item.hash = l.getDataAt(1);
				item.size = "--";

				String content = l.getDataAt(2);
				toWrite.add(item);

				CutResult cutted = Passage.cutRawContent(content, item.name, info.maxLen);
				appendLogs(cutted.logs());
				for (String part : cutted)
				{
					SubItem sub = new SubItem(item);
					item.subItems.add(sub);
					sub.content = part;
				}
			}
		}

		parseOutput(type, toWrite, output);
	}

	int parseFromFolder(int type, File folder, File output, boolean useLegacy, boolean keepLegacy)
			throws NoSuchAlgorithmException
	{
		appendLog("************************");
		appendLog("Folder parse job for [" + folder.getPath() + "]");
		appendLog("Output to: " + output.getParent());
		appendLog("Use legacy: " + useLegacy);
		appendLog("Keep extra legacy: " + keepLegacy);

		if (useLegacy && output.exists())
			loadLegacy(type, output);
		else
			appendLog(LOGGER.log(LOGGER.APP, "Skip load legacy file."));

		List<Item> toWrite = new ArrayList<>();

		File[] files = folder.listFiles();

		if (files == null || files.length == 0)
		{
			appendLog(LOGGER.log(LOGGER.APP, "Empty source folder: " + folder.getPath()));
			return 0;
		}

		appendLog(LOGGER.log(LOGGER.APP, "Loading " + files.length + " files from source folder."));

		int parseNum = 0;
		int legacyNum = 0;
		int skip = 0;

		for (File file : files)
		{
			if (file.isDirectory())
			{
				skip++;
				continue;
			}

			if (!file.exists())
			{
				LOGGER.log(LOGGER.APP, "File not found: " + file.getAbsolutePath());
				continue;
			}

			String hash = FileUtility.fileContentHash(file);
			String name = file.getName();
			String content = LineContent.loadString(file);
			long size = file.length();

			if (!StrUtility.stringValid(content))
			{
				skip++;
				continue;
			}

			String key = name + hash;
			Item legacy = itemMap.get(key);

			if (legacy == null)
			{
				Item item = new Item();
				item.name = name;
				item.hash = hash;
				toWrite.add(item);

				CutResult cutted = Passage.cutRawContent(content, name, info.maxLen);
				appendLogs(cutted.logs());

				for (String cut : cutted)
				{
					SubItem sub = new SubItem(item);
					item.subItems.add(sub);
					sub.content = cut;
				}

				item.size = FileUtility.getFileSizeString(size);
				parseNum++;
			} else
			{
				toWrite.add(legacy);
				legacyNum++;
			}

			itemMap.remove(key);
		}

		appendLog(LOGGER.log(LOGGER.APP, "Loading files done. " + parseNum + " files to parse; " + legacyNum
				+ " files use buffed legacy; " + skip + " unusing files skipped."));

		if (keepLegacy)
		{
			appendLog(LOGGER.log(LOGGER.APP, "Append " + itemMap.size() + " items from legacy data."));

			List<Item> legacyItems = new ArrayList<>(itemMap.values());
			legacyItems.addAll(toWrite);
			toWrite = legacyItems;
		}

		return parseOutput(type, toWrite, output);
	}

	int parseOutput(int type, List<Item> toWrite, File output)
	{
		int count = 0;

		flushLog();
		int total = toWrite.size();
		int full = 0;
		int keep = 0;
		for (Item i : toWrite)
		{
			for (SubItem sub : i.subItems)
			{
				if (sub.qaList.isEmpty())
					full++;
				else
					keep++;
			}
		}

		appendLog(LOGGER.log(LOGGER.APP,
				"Begin parse data. Items: " + total + ". Total sub parse: " + full + ". Legacy: " + keep), true);

		Workbook wb = makeXlsxFile(output, toWrite.size());

		StyleGroup sg = new StyleGroup(wb, null);

		Sheet infoSheet = wb.createSheet(TAB_INFO);
		appendLine(infoSheet, INFO_NUM, INFO_LEN);
		appendLine(infoSheet, info.num, info.maxLen);
		infoSheet.setColumnWidth(0, 5000);
		infoSheet.setColumnWidth(1, 5000);

		Sheet sheet = wb.createSheet(TAB_SHEET);
		String[] head =
		{ KEY_NAME, KEY_HASH, KEY_CONTENT, KEY_LEN, KEY_RESPONSE, KEY_SPLIT, KEY_Q, KEY_A, KEY_COST };
		addLine(sheet, head, 0);

		for (int i = 0; i < head.length; i++)
		{
			switch (head[i])
			{
			case KEY_SPLIT:
			case KEY_LEN:
				sheet.setColumnWidth(i, 2000);
				break;
			default:
				sheet.setColumnWidth(i, 5000);
				break;
			}
		}
		int checked = 1;
		int subs = 1;
		int netExp = 0;

		boolean breakLimit = false;
		saveTick.clear();

		loopWrite: for (Item item : toWrite)
		{
			boolean autoSave = false;
			try
			{
				String head1 = "[" + checked + "/" + total + "]";

				for (SubItem sub : item.subItems)
				{
					if (!StrUtility.stringValid(sub.response))
					{
						if (info.parseLimit > 0 && count >= info.parseLimit)
						{
							if (!breakLimit)
							{
								breakLimit = true;
								LOGGER.log(LOGGER.APP, "Break parsing at [" + subs + "/" + full
										+ "] since reach parse number limit: " + info.parseLimit);
							}

							fillEmptySp(info.num, sub.qaList);
						} else
						{
							String head2 = head1 + "[" + subs + "/" + full + "] parsing: " + sub;
							LOGGER.log(LOGGER.APP, head2);
							subs++;
							sub.remotecall(type, info.num, head2);
							autoSave = true;
							count++;
						}
					} else if (detailLog)
						LOGGER.log(LOGGER.APP, head1 + " use legacy data: " + item);
				}
			} catch (Exception e)
			{
				netExp++;
				e.printStackTrace();
				appendErrLog(LOGGER.log(LOGGER.APP, "Server exception while item " + item));

				if (info.breatNetException)
				{
					if (writeCount > 0)
					{
						appendErrLog(LOGGER.log(LOGGER.APP, "Save file and stop parse job at [" + checked + "/" + total + "]"));
						saveXlsFile(wb, output);
						writeCount = 0;
					} else
						appendErrLog(LOGGER.log(LOGGER.APP, "Stop parse job at [" + checked + "/" + total + "]"));

					flushLog();
					break loopWrite;
				} else
					LOGGER.log(LOGGER.APP, "Continue after " + netExp + " server exceptions.");
			}

			writeItem(wb, output, sheet, item, sg, autoSave);

			if (checked % 100 == 0)
			{
				System.out.println(checked + "/" + total + " items parsed.");
			}
			checked++;
		}

		if (writeCount > 0)
			saveXlsFile(wb, output);
		flushLog();

		String fname = FileUtility.getFileName(output) + "_[" + sheet.getLastRowNum() + "] at "
				+ dateFormat.format(new Date(System.currentTimeMillis())) + ".xlsx";
		File cache = new File(output.getParent() + "/cache/" + fname);
		saveXlsFile(wb, cache, true);

		return count;
	}

	TimeTicker saveTick = new TimeTicker();

	void writeItem(Workbook wb, File output, Sheet sheet, Item item, StyleGroup sg, boolean autoSave)
	{
		for (SubItem sub : item.subItems)
		{
			boolean first = true;

			for (StringPair sp : sub.qaList)
			{
				if (first)
				{
					CellStyle upLine;
					Row row;

					String name = "";
					String hash = "";

					if (sub.index == 0)
					{
						upLine = sg.style(BaseStyle.BORDER_W_UP);
						name = item.name;
						hash = item.hash;
					} else
					{
						upLine = sg.style(BaseStyle.BORDER_UP);
					}
					row = appendLine(sheet, name, hash, sub.content, sub.content.length(), sub.response, sub.state, sp.getKey(),
							sp.getValue(), sub.cost);

					List<Cell> cells = CollUtility.iteratorToList(row.cellIterator());
					for (Cell cell : cells)
					{
						if (cell != null)
							cell.setCellStyle(upLine);
					}

					first = false;
				} else
				{
					appendLine(sheet, "", "", "", "", "", "", sp.getKey(), sp.getValue(), "");
				}
			}

			writeCount++;
		}

		if (autoSave && writeBuff > 0 && writeCount > writeBuff)
		{
			writeCount = 0;
			appendLog(LOGGER.log(LOGGER.FILE,
					"Save output file for every " + writeBuff + " items parsed. Cost: " + saveTick.tickS()));
			saveXlsFile(wb, output);
			flushLog();
		}
	}

	static void fillEmptySp(int num, List<StringPair> holder)
	{
		if (holder == null)
			return;

		while (holder.size() < num)
		{
			holder.add(StringPair.empty());
		}
	}

	public List<StringPair> splitContent(int type, String input, int num, String info)
	{
		ResponseSplitter splitter = new ResponseSplitter(num, false);
		SplitResult result = splitter.splitResponse(input, type);

		List<StringPair> spList = new ArrayList<>();

		if (result.target() == null)
		{
			fillEmptySp(num, spList);
			return spList;
		}

		if (remover == null)
			spList.addAll(result.target());
		else
		{
			for (StringPair sp : result.target())
			{
				spList.add(remover.replacePair(sp));
			}
		}

		String err = result.info();

		if (StrUtility.stringValid(err))
		{
			if (result.size() == 0)
			{
				appendLog(err, true);
			} else if (result.size() < num)
			{
				if (detailLog)
					appendLog(err, true);
			}
		}

		fillEmptySp(num, spList);

		return spList;
	}

	void loadLegacy(int type, File legacy)
	{
		XlsData pages = ExcelReader.read(legacy);

		Page infoPage = null;
		Page dataPage = null;

		for (Page p : pages)
		{
			if (p.getName().equals(TAB_INFO))
				infoPage = p;
			if (p.getName().equals(TAB_SHEET))
				dataPage = p;
		}

		int num;
		int len;

		try
		{
			Line infoLine = infoPage.getLines().get(0);

			num = infoLine.getFieldInteger(INFO_NUM, -1);
			if (num != info.num)
				return;

			len = infoLine.getFieldInteger(INFO_LEN, -1);
			if (len != info.maxLen)
				return;

		} catch (Exception e)
		{
			return;
		}

		if (dataPage == null)
			return;

		Item current = null;
		SubItem sub = null;
		for (Line l : dataPage.getLines())
		{
			String name = l.getFieldData(KEY_NAME);
			String hash = l.getFieldData(KEY_HASH);

			if (StrUtility.stringValid(name) && StrUtility.stringValid(hash))
			{
				String key = name + hash;
				current = itemMap.get(key);
				if (current == null)
				{
					current = new Item();
					itemMap.put(key, current);
					current.name = name;
					current.hash = hash;
				}
			}

			if (current != null)
			{
				String content = l.getFieldData(KEY_CONTENT);

				if (StrUtility.stringValid(content))
				{
					sub = new SubItem(current);
					sub.content = content;
					current.subItems.add(sub);
					sub.cost = l.getFieldData(KEY_COST);
					sub.response = l.getFieldData(KEY_RESPONSE);
					sub.state = l.getFieldData(KEY_SPLIT);
				}

				if (sub != null)
				{
					String q = l.getFieldData(KEY_Q);
					String a = l.getFieldData(KEY_A);
					sub.qaList.add(new StringPair(q, a));
				}
			}
		}

		for (Item i : itemMap.values())
		{
			for (SubItem s : i.subItems)
			{
				if (!StrUtility.stringValid(s.state))
					s.checkState(num);
				else if (s.state.equals("Empty") && StrUtility.stringValid(s.response))
				{
					s.qaList = splitContent(type, s.response, info.num, "");
					s.checkState(num);
				}

			}
		}

		appendLog(LOGGER.log(LOGGER.APP, "Load " + itemMap.size() + " items from legacy file."));
	}

	static int writeBuff = 10;
	static int logBuff = 5;
	static PerfixRemover remover;

	@SuppressWarnings("unused")
	public static void main(String[] args)
	{
		JobInfo info = new JobInfo();
		TimeTicker ticker = new TimeTicker();

		info.logFile = new File("V:/doc qa/log.txt");

		info.url = GptJobConfig.getProp("url_a10x8") + ":7666";

		QaParseJob job = new QaParseJob(info);

		File source;
		File output;

		remover = new PerfixRemover(new File("V:/doc qa/remove_perfix.txt"));

		// source = new File("V:/doc qa/政策类知识信息数据-处理后/test/");
		// output = new File("V:/doc qa/parse_output_test.xlsx");
		// job.parseFromFolder(source, output, false, false);

		// info.maxLen = 10000;
		// source = new File("V:/doc qa/政策类知识信息数据-处理后/large/");
		// output = new File("V:/doc qa/parse_output_large.xlsx");
		// job.parseFromFolder(source, output, false, false);

		info.maxLen = 2000;
		info.num = 1;
		info.parseLimit = 1000;
		// skipParse = true;
		detailLog = false;
		source = new File("V:/doc qa/doc_contents/");
		output = new File("V:/doc qa/parse_Chose_output_1_2000.xlsx");

		/**
		 * Type: 1 - QA, 2 - Choice, 3 - TF, 4 - Fill
		 */

		try
		{
			job.parseFromFolder(2, source, output, true, false);
		} catch (NoSuchAlgorithmException e)
		{
			e.printStackTrace();
		}

		// source = new File("V:/doc qa/2023年口径库相关知识点.xlsx");
		// output = new File("V:/doc qa/parse_output_2023.xlsx");
		// job.parseFromForm(source, output);

		// splitTest(new File("v:/doc qa/temp.txt"));
		// splitTest(new File("v:/doc qa/temp2.txt"));
		// splitTest(new File("v:/doc qa/temp3.txt"));

		System.out.println("All done. Total time cost: " + ticker.tickS());
	}

	static boolean skipParse = false;
	static boolean detailLog = true;

	@SuppressWarnings("unused")
	static void splitTest(File file)
	{
		QaParseJob job = new QaParseJob(new JobInfo());
		String content = LineContent.loadString(file);

		int type = 1;
		List<StringPair> sps = job.splitContent(type, content, 3, "Temp");
		System.out.println("=====================");
		for (StringPair sp : sps)
		{
			System.out.println("---------");
			System.out.println("Q:\t" + sp.first());
			System.out.println("*******");
			System.out.println("A:\t" + sp.second());
		}
		System.out.println("******* done");
	}
}
